<html>
  <head>
    <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>DistCache: Provable Load Balancing for LargeScale Storage Systems with Distributed Caching | Elvis Zhang</title>
<meta name="description" content="The easy way or the right way." />
<link rel="shortcut icon" href="https://blog.shunzi.tech/favicon.ico">
<link rel="stylesheet" href="https://blog.shunzi.tech/styles/main.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.10.0/katex.min.css">

<script data-ad-client="ca-pub-7661668224317940" async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
<script src="https://blog.shunzi.tech/media/js/jquery.min.js"></script>
<script src="https://blog.shunzi.tech/media/js/masonry.pkgd.min.js"></script>
<script src="https://blog.shunzi.tech/media/js/aos.js"></script>
<script src="https://blog.shunzi.tech/media/js/pace.min.js"></script>
<script src="https://blog.shunzi.tech/media/js/view-image.min.js"></script>
<script src="https://blog.shunzi.tech/media/js/jquery.magnific-popup.min.js"></script>
<script src="https://blog.shunzi.tech/media/js/functions.js"></script>
    <meta name="referrer" content="never">
    <meta name="description" content="

FAST19 Best Paper DistCache: Provable Load Balancing for LargeScale Storage Systems with Distributed Caching





FAST..." />
    <meta name="keywords" content="缓存" />
    <script src="https://blog.shunzi.tech/media/js/waterfall.min.js"></script>
    <script src="https://blog.shunzi.tech/media/js/prism.min.js"></script>
  </head>
  <body>
            <header id="header" class="grid-container">
        <!-- start: .menu-wrapper -->
        <div class="menu-mobile"> 
          <i class="fa fa-reorder"></i>
        </div>
        <div class="menu-wrapper">
          <div class="">
            <div class="logo">
              <a href="https://blog.shunzi.tech"><img src="\media\images\custom-headerLogo.jpg" alt=""></a>
            </div>
            <!-- start: .main-nav -->

            <nav class="main-nav grid-container grid-parent">
              <ul id="menu-header" class="menu gradient-effect">
                <li class=""><a href="https://blog.shunzi.tech" class="menu">首页</a></li>
                
                  <li class="" >
                    <a href="/archives" class="menu">
                      归档
                    </a>
                  </li>
                
                  <li class="" >
                    <a href="/tag/diary" class="menu">
                      随笔
                    </a>
                  </li>
                
                  <li class="" >
                    <a href="/movies" class="menu">
                      观影
                    </a>
                  </li>
                
                  <li class="" >
                    <a href="/post/about" class="menu">
                      关于
                    </a>
                  </li>
                
                <li class="search-menu-item hide-on-mobile hide-on-tablet"><a href="#search-lightbox" class="lightbox mfp-inline"><i class="fa fa-search-line"></i></a></li>
              </ul>
            </nav>
            <a href="#search-lightbox" class="lightbox epcl-search-button mfp-inline hide-on-tablet hide-on-desktop"><i class="fa fa-search-line"></i></a>
            <!-- end: .main-nav -->
            <div class="clear"></div>
            <div class="border hide-on-tablet hide-on-mobile"></div>
          </div>    
          <div class="clear"></div>
        </div>
        <!-- end: .menu-wrapper -->
        <div class="clear"></div>
      </header>
      <div class="hide-on-mobile hide-on-tablet hide-on-desktop">
        <div id="search-lightbox" class="grid-container grid-small grid-parent mfp-hide">
          <div class="search-wrapper section">
            <form id="gridea-search-form" data-update="1620954331293" action="/search/index.html" class="search-form" _lpchecked="1">
              <input type="text" name="q" id="s" value="" class="search-field" placeholder="搜点啥..." aria-label="搜点啥..." required="">
              <button type="submit" class="submit" aria-label="Submit">
                <i class="fa fa-search-line"></i>
              </button>
            </form>
          </div>
        </div>
      </div>

      <main id="single" class="main grid-container fullcover no-sidebar aos-init aos-animate" data-aos="fade">

        <div class="center content">
          <div class="featured-image cover" style="background-image: url('https://blog.shunzi.tech/post-images/distcache-provable-load-balancing-for-largescale-storage-systems-with-distributed-caching.png');">
            <div class="meta top"> 
              <time class="meta-info" style="float:left;" datetime="2020-05-05"><i class="fa fa-calendar"></i><span class="lately">1 年前</span></time>
              
              <a href="https://blog.shunzi.tech/post/distcache-provable-load-balancing-for-largescale-storage-systems-with-distributed-caching/#comments" class="comments meta-info" title="">
                <i class="fa fa-comment remixicon"></i><span class="comment-count valine-comment-count" data-xid="/distcache-provable-load-balancing-for-largescale-storage-systems-with-distributed-caching/"> </span>
              </a>
              <span id="/distcache-provable-load-balancing-for-largescale-storage-systems-with-distributed-caching/" class="leancloud_visitors views-counter meta-info" title=""><i class="fa fa-leancloud remixicon"></i><span class="leancloud-visitors-count"></span></span>
              
            </div>
            <div class="info">
              <div class="tags ">
                
                      <a href="https://blog.shunzi.tech/tag/9msH-lUaA/" class="ctag ctag-0 ctag-9msH-lUaA" aria-label="">缓存</a>
                    
              </div>
              <h1 class="title ularge white bold">DistCache: Provable Load Balancing for LargeScale Storage Systems with Distributed Caching</h1>
            </div>
          </div>
        </div>  

        <div class="epcl-page-wrapper">
          <div class="left-content grid-70 np-mobile">
            <article class="main-article post">
              <section class="post-content">
                <div class="text">
                  <blockquote>
<ul>
<li>FAST19 Best Paper DistCache: Provable Load Balancing for LargeScale Storage Systems with Distributed Caching</li>
</ul>
</blockquote>
<!--more-->
<blockquote>
<ul>
<li>FAST19 Best Paper DistCache: Provable Load Balancing for LargeScale Storage Systems with Distributed Caching</li>
<li>BestPaper 以及有很多大佬都对该文章有相应的解读，所以想看下原文，恰巧现阶段的项目中也有涉及分布式缓存的东西，学习一下相关思想。</li>
<li>主要设计了一个保证存储系统负载均衡的分布式缓存，并提供了相关理论证明</li>
<li>由于相关解读资料很多，此处只简单记录一些思想和关键点。也是一种新的阅读论文方式的尝试。</li>
</ul>
</blockquote>
<h2 id="abstract">Abstract</h2>
<ul>
<li>负载均衡是大规模存储系统需要实现的功能，从而达到严苛的 SLO(service-level objectives)</li>
<li>实现负载均衡的方式通常为建立缓存系统，而传统的缓存系统方式往往会有缓存节点数据不均衡或者一致性开销较高的问题。</li>
<li>本文提出的 DistCache机制，使用 independent hash function 做 item partition，query 则受启发于 power-of-two-choices。</li>
</ul>
<h2 id="introduction">Introduction</h2>
<ul>
<li>大规模存储系统在 scale out 时一个最大的问题就是 负载均衡。具体表现则是，部分节点的数据被频繁访问，导致该节点可能出现过载，以及产生热点数据，从而过载的节点将成为整个系统的瓶颈，可能导致高尾延迟、低吞吐量。</li>
<li>负载均衡的解决方案通常都是使用缓存。而理论证明（SOCC2011 Small cache, big effect: Provable load balancing for randomly partitioned cluster services）,n个存储节点，无论query是怎样的分布，缓存最热的O(nlogn)个对象即可均衡负载。即缓存大小只是和存储节点的数目相关，和存储节点上存储的对象数无关。相关论文有 SwitchKV（NSDI 2016），NetCache（SOSP 2017）</li>
<li>但是上面的小Cache解决热点的方案不能很好地scale out到多个集群，原因是一个集群一个Cache节点只能保证集群内的负载均衡，而集群间的负载均衡无法保证。为了保证集群间的负载均衡，最简单的方式则是再添加一个缓存节点来保证每个集群的负载均衡，但此时整个集群的吞吐量将受限于新添加的缓存节点，不足以支撑多个集群的吞吐，因此集群间需要一层分布式的Cache。而分布式Cache的难点在于如果简单地replicate热点数据到多个Cache节点会有一致性问题，而简单地按照hash partition又会造成Cache Node之间的负载不均衡导致吞吐受限于一个节点。所以关键就是避免Cache Node间的不均衡以及保证Cache一致性的开销。</li>
<li>提出了分布式缓存 DistCache。核心思想：使用独立的 HASH 函数在不同层的缓存节点之间对热数据进行分区，并将 The Power of Two Random Choices 应用到自适应的数据路由算法中。</li>
</ul>
<h5 id="参考链接">参考链接</h5>
<ul>
<li><a href="https://zhuanlan.zhihu.com/p/64538762">[知乎]：The Power of Two Random Choices</a></li>
<li><a href="https://zhuanlan.zhihu.com/p/66273612">[知乎]：Small Cache, Big Effect</a></li>
</ul>
<h2 id="background">Background</h2>
<ul>
<li>同时保证集群内部和集群间负载均衡的方案：<br>
<img src="https://blog.shunzi.tech/post-images/1588652495483.png" alt="" loading="lazy"></li>
<li>图b 关键问题：如何将热数据分配到分布式缓存层？
<ul>
<li>缓存分区：多个热数据可能被分区到同一个上层缓存节点上，导致上层缓存节点之间负载不均衡，使得缓存吞吐量不能线性增长，会受限于负载最高的节点。</li>
<li>缓存复制：虽然保证了负载均衡，但引入了较大的缓存一致性开销，需要更新上层缓存节点上所有副本。</li>
</ul>
</li>
<li>故需要实现：避免上层缓存节点之间的负载不均现象，并减少缓存一致性的开销。</li>
</ul>
<h2 id="design">Design</h2>
<ul>
<li>缓存分配时，上下两层采用不同的 HASH 算法，如图所示 C3 过载时，其热点数据将会均匀分布在上层缓存节点 C0,C1,C2，上层节点将主要吸收对于热点数据的 IO 操作。</li>
<li>缓存分配机制只是提供了一种负载均衡的方式，但未提供查询的分布方法。故 DistCache 方案采用了 The Power of Two Random Choices，如图所示，GET A 操作将被路由到 C1 或者 C3 节点，根据 C1 和 C3 的负载决定最终的选择。从而无需依赖中心节点来统计集群信息。<br>
<img src="https://blog.shunzi.tech/post-images/1588661091612.png" alt="" loading="lazy"></li>
<li>对于缓存的更新操作，则是保证缓存数据一致性的常用方案。DistCache 使用了两阶段更新协议，即在更新 Cache 数据之前，标记老数据无效，然后首先更新 server 上的数据，再更新 cache 数据。</li>
</ul>
<h3 id="理论证明推导">理论证明推导</h3>
<ul>
<li>详见论文。</li>
</ul>
<h2 id="instance">Instance</h2>
<h3 id="switch-based-caching">Switch-Based Caching</h3>
<ul>
<li>基于可编程的交换机缓存系统的设计</li>
</ul>
<h4 id="architecture">Architecture</h4>
<figure data-type="image" tabindex="1"><img src="https://blog.shunzi.tech/post-images/1588673983947.png" alt="" loading="lazy"></figure>
<h4 id="component">Component</h4>
<ul>
<li><strong>Controller</strong>：用于计算缓存分区，并通知缓存交换机。它更新系统重新配置下的缓存分配。</li>
<li><strong>Cache switches</strong>：
<ul>
<li>功能1：缓存热点数据</li>
<li>功能2：分发用于查询路由的交换机负载信息。</li>
</ul>
</li>
<li><strong>ToR switches at client racks</strong>： 提供查询路由</li>
<li><strong>Storage servers</strong>：使用开源存储软件进行 KV 数据的存储，同时实现缓存数据一致性。</li>
</ul>
<h4 id="读流程">读流程</h4>
<ul>
<li>R3 -&gt; S6 -&gt; Upper layer(S0, S1) -&gt; S1 hit</li>
<li>R3 -&gt; S6 -&gt; Upper layer(S0, S1) -&gt; S0 -&gt; S3 hit<br>
<img src="https://blog.shunzi.tech/post-images/1588674609700.png" alt="" loading="lazy"></li>
</ul>
<h4 id="写流程">写流程</h4>
<figure data-type="image" tabindex="2"><img src="https://blog.shunzi.tech/post-images/1588674874812.png" alt="" loading="lazy"></figure>
<h2 id="evaluation">Evaluation</h2>
<h3 id="读性能">读性能</h3>
<ul>
<li>读性能表现 DistCache 与 Cache Replication 一样好<br>
<img src="https://blog.shunzi.tech/post-images/1588675071396.png" alt="" loading="lazy"></li>
</ul>
<h3 id="写性能">写性能</h3>
<ul>
<li>DistCache 在写占比较低的时候能够保证高吞吐</li>
<li>Cache Replication 一致性开销较大，随着写占比上升，吞吐量下降的较快。<br>
<img src="https://blog.shunzi.tech/post-images/1588675267513.png" alt="" loading="lazy"></li>
</ul>
<h4 id="参考链接-2">参考链接</h4>
<ul>
<li><a href="https://jimchenglin.github.io/2019/04/15/FAST-2019-DistCache/">[1] FAST 2019 DistCache</a></li>
</ul>

                </div>
                <div class="clear"></div>
              </section>
            </article>
            <div class="clear"></div>

            <section class="related section">
              
              <article class="prev grid-50 tablet-grid-50 grid-parent">
                <div class="thumb cover lazy loaded" style="background-image: url('https://blog.shunzi.tech/post-images/the-log-structured-merge-tree-lsm-tree.png');"></div>
                 <a href="https://blog.shunzi.tech/post/the-log-structured-merge-tree-lsm-tree/" class="full-link"></a>
                 <div class="info">
                  <time datetime="2020-05-08">2020-05-08</time>
                  <h4 class="title white no-margin">The Log-Structured Merge-Tree (LSM-Tree)</h4>
                </div>
                 <span class="epcl-button red">
                  <img src="https://blog.shunzi.tech/media/images/left-arrow.svg" width="15" alt="Left Arrow">
                </span>
                <div class="overlay"></div>
              </article>
              
              
              <article class="next grid-50 tablet-grid-50 grid-parent">
                <div class="thumb cover lazy loaded" style="background-image: url('https://blog.shunzi.tech/post-images/cpp-basic.jpg');"></div>
                 <a href="https://blog.shunzi.tech/post/cpp-basic/" class="full-link"></a>
                 <div class="info">
                  <time datetime="2020-04-06">2020-04-06</time>
                  <h4 class="title white no-margin">C++基础快速入门</h4>
                </div>
                 <span class="epcl-button red">
                  <img src="https://blog.shunzi.tech/media/images/right-arrow.svg" width="15" alt="Left Arrow">
                </span>
                <div class="overlay"></div>
              </article>
              

                <div class="clear"></div>
            </section>

              <div class="clear"></div>
              
            
              <div id="comments" class="bg-white hosted ">
                <div class="clear"></div>
<script>
jQuery(document).ready(function($){
    $('.vemoji-btn').text('😀');
    $("#comments").on('click', 'span.vat',function(){
        $(this).parent('div.vmeta').next("div.vcontent").after($("div.vwrap"));
        $('textarea#veditor').focus();
    })
    if(window.location.hash){
        var checkExist = setInterval(function() {
            if ($(window.location.hash).length) {
                $('html, body').animate({scrollTop: $(window.location.hash).offset().top-200}, 600);
                clearInterval(checkExist);
            }
        }, 100);
    }
})
</script>

              </div>
            

            </div>
          </div>
      </main>

          <footer id="footer" class="grid-container">
        <div class="widgets row gradient-effect">
            <div class="default-sidebar border-effect">
              <div class="grid-33 tablet-grid-50 mobile-grid-100">
                <section id="tag_cloud-2" class="widget widget_epcl_posts_thumbs underline-effect">
                  <h4 class="widget-title title white bordered">最新文章</h4>
                  
                  
                  <article class="item post-0 post type-post status-publish format-standard has-post-thumbnail hentry">
                    <a href="https://blog.shunzi.tech/post/cpp-multi-thread/" class="thumb hover-effect">
                      <span class="fullimage cover" style="display:block;border-radius:50%;background-image: url('https://raw.githubusercontent.com/zjs1224522500/PicGoImages/master//img/blog/20210513192958.png');"></span>
                    </a>
                    <div class="info gradient-effect">
                      <time datetime="2021-05-06">2021-05-06</time>
                      <h4 class="title usmall">
                        <a href="https://blog.shunzi.tech/post/cpp-multi-thread/">C++ 多线程</a>
                      </h4>
                    </div>
                    <div class="clear"></div>
                  </article>
                  
                  
                  
                  <article class="item post-1 post type-post status-publish format-standard has-post-thumbnail hentry">
                    <a href="https://blog.shunzi.tech/post/c-basic/" class="thumb hover-effect">
                      <span class="fullimage cover" style="display:block;border-radius:50%;background-image: url('https://raw.githubusercontent.com/zjs1224522500/PicGoImages/master//img/blog/20210513192631.png');"></span>
                    </a>
                    <div class="info gradient-effect">
                      <time datetime="2021-04-06">2021-04-06</time>
                      <h4 class="title usmall">
                        <a href="https://blog.shunzi.tech/post/c-basic/">C 基础</a>
                      </h4>
                    </div>
                    <div class="clear"></div>
                  </article>
                  
                  
                  
                  <article class="item post-2 post type-post status-publish format-standard has-post-thumbnail hentry">
                    <a href="https://blog.shunzi.tech/post/basic-of-concurrency-one/" class="thumb hover-effect">
                      <span class="fullimage cover" style="display:block;border-radius:50%;background-image: url('https://raw.githubusercontent.com/zjs1224522500/PicGoImages/master//img/blog/20200717213648.png');"></span>
                    </a>
                    <div class="info gradient-effect">
                      <time datetime="2021-04-05">2021-04-05</time>
                      <h4 class="title usmall">
                        <a href="https://blog.shunzi.tech/post/basic-of-concurrency-one/">Series Three of Basic of Concurrency - Condition Variables</a>
                      </h4>
                    </div>
                    <div class="clear"></div>
                  </article>
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  
                  <div class="clear"></div>
                </section>
              </div>

              <div class="grid-33 tablet-grid-50 mobile-grid-100">
                <section id="tag_cloud-2" class="widget widget_tag_cloud underline-effect">
                  <h4 class="widget-title title white bordered">标签云</h4>
                  <div class="tagcloud">
                    
                      <a href="https://blog.shunzi.tech/tag/n2w6bz87h/" class="ctag ctag-0 ctag-n2w6bz87h" aria-label="">编程语言</a>
                    
                      <a href="https://blog.shunzi.tech/tag/3zCwFWPHxH/" class="ctag ctag-1 ctag-3zCwFWPHxH" aria-label="">存储</a>
                    
                      <a href="https://blog.shunzi.tech/tag/la-n8a0mo/" class="ctag ctag-2 ctag-la-n8a0mo" aria-label="">读书笔记</a>
                    
                      <a href="https://blog.shunzi.tech/tag/os/" class="ctag ctag-3 ctag-os" aria-label="">OS</a>
                    
                      <a href="https://blog.shunzi.tech/tag/5uQUdLlSC/" class="ctag ctag-4 ctag-5uQUdLlSC" aria-label="">Paper</a>
                    
                      <a href="https://blog.shunzi.tech/tag/_jfuTNqah/" class="ctag ctag-5 ctag-_jfuTNqah" aria-label="">LSM</a>
                    
                      <a href="https://blog.shunzi.tech/tag/hbaTDSglx-/" class="ctag ctag-6 ctag-hbaTDSglx-" aria-label="">工具</a>
                    
                      <a href="https://blog.shunzi.tech/tag/EO3XpMf_y/" class="ctag ctag-7 ctag-EO3XpMf_y" aria-label="">Linux</a>
                    
                      <a href="https://blog.shunzi.tech/tag/wAFV_pvXZ/" class="ctag ctag-8 ctag-wAFV_pvXZ" aria-label="">cs-course</a>
                    
                      <a href="https://blog.shunzi.tech/tag/VqiGqmxbod/" class="ctag ctag-9 ctag-VqiGqmxbod" aria-label="">6.824</a>
                    
                      <a href="https://blog.shunzi.tech/tag/geK0jEW-T/" class="ctag ctag-10 ctag-geK0jEW-T" aria-label="">分布式</a>
                    
                      <a href="https://blog.shunzi.tech/tag/l8sKsLUAi/" class="ctag ctag-11 ctag-l8sKsLUAi" aria-label="">KVS</a>
                    
                      <a href="https://blog.shunzi.tech/tag/9msH-lUaA/" class="ctag ctag-12 ctag-9msH-lUaA" aria-label="">缓存</a>
                    
                      <a href="https://blog.shunzi.tech/tag/i2b42Y2j6/" class="ctag ctag-13 ctag-i2b42Y2j6" aria-label="">Ceph</a>
                    
                      <a href="https://blog.shunzi.tech/tag/oBVOD8v4ou/" class="ctag ctag-14 ctag-oBVOD8v4ou" aria-label="">一致性</a>
                    
                      <a href="https://blog.shunzi.tech/tag/gqgftpk_y/" class="ctag ctag-15 ctag-gqgftpk_y" aria-label="">AI</a>
                    
                      <a href="https://blog.shunzi.tech/tag/shu-ju-ku/" class="ctag ctag-16 ctag-shu-ju-ku" aria-label="">数据库</a>
                    
                      <a href="https://blog.shunzi.tech/tag/ZnIN9Ge-w/" class="ctag ctag-17 ctag-ZnIN9Ge-w" aria-label="">对象存储</a>
                    
                      <a href="https://blog.shunzi.tech/tag/4zx4ysLGro/" class="ctag ctag-18 ctag-4zx4ysLGro" aria-label="">云计算</a>
                    
                      <a href="https://blog.shunzi.tech/tag/Y_nsOD1At/" class="ctag ctag-19 ctag-Y_nsOD1At" aria-label="">SSD</a>
                    
                      <a href="https://blog.shunzi.tech/tag/E2d1yYZcV8/" class="ctag ctag-20 ctag-E2d1yYZcV8" aria-label="">虚拟化</a>
                    
                      <a href="https://blog.shunzi.tech/tag/PhD/" class="ctag ctag-21 ctag-PhD" aria-label="">Ph.D</a>
                    
                      <a href="https://blog.shunzi.tech/tag/ZqEqvRTvl/" class="ctag ctag-22 ctag-ZqEqvRTvl" aria-label="">网络</a>
                    
                      <a href="https://blog.shunzi.tech/tag/PuY19cs53/" class="ctag ctag-23 ctag-PuY19cs53" aria-label="">仿真</a>
                    
                      <a href="https://blog.shunzi.tech/tag/rIIc9E-ZvN/" class="ctag ctag-24 ctag-rIIc9E-ZvN" aria-label="">系统结构</a>
                    
                      <a href="https://blog.shunzi.tech/tag/fu-wu-qi/" class="ctag ctag-25 ctag-fu-wu-qi" aria-label="">服务器</a>
                    
                      <a href="https://blog.shunzi.tech/tag/X-lnqf1Ex/" class="ctag ctag-26 ctag-X-lnqf1Ex" aria-label="">容器</a>
                    
                      <a href="https://blog.shunzi.tech/tag/5h7k39FKw/" class="ctag ctag-27 ctag-5h7k39FKw" aria-label="">C语言</a>
                    
                      <a href="https://blog.shunzi.tech/tag/diary/" class="ctag ctag-28 ctag-diary" aria-label="">Diary</a>
                    
                      <a href="https://blog.shunzi.tech/tag/DyzFtOe6x/" class="ctag ctag-29 ctag-DyzFtOe6x" aria-label="">计算机基础</a>
                    
                      <a href="https://blog.shunzi.tech/tag/oqE3oKihb/" class="ctag ctag-30 ctag-oqE3oKihb" aria-label="">OpenStack</a>
                    
                      <a href="https://blog.shunzi.tech/tag/p_z7gKe6R/" class="ctag ctag-31 ctag-p_z7gKe6R" aria-label="">中间件</a>
                    
                      <a href="https://blog.shunzi.tech/tag/Test/" class="ctag ctag-32 ctag-Test" aria-label="">测试</a>
                    
                      <a href="https://blog.shunzi.tech/tag/Product-Standard/" class="ctag ctag-33 ctag-Product-Standard" aria-label="">Product Standard</a>
                    
                      <a href="https://blog.shunzi.tech/tag/spring/" class="ctag ctag-34 ctag-spring" aria-label="">Spring</a>
                    
                      <a href="https://blog.shunzi.tech/tag/she-ji-mo-shi/" class="ctag ctag-35 ctag-she-ji-mo-shi" aria-label="">设计模式</a>
                    
                      <a href="https://blog.shunzi.tech/tag/mian-jing/" class="ctag ctag-36 ctag-mian-jing" aria-label="">面经</a>
                    
                      <a href="https://blog.shunzi.tech/tag/suan-fa/" class="ctag ctag-37 ctag-suan-fa" aria-label="">算法</a>
                    
                      <a href="https://blog.shunzi.tech/tag/redis/" class="ctag ctag-38 ctag-redis" aria-label="">Redis</a>
                    
                      <a href="https://blog.shunzi.tech/tag/javaweb/" class="ctag ctag-39 ctag-javaweb" aria-label="">JavaWeb</a>
                    
                      <a href="https://blog.shunzi.tech/tag/KyMCZj2Wl/" class="ctag ctag-40 ctag-KyMCZj2Wl" aria-label="">WEB容器</a>
                    
                      <a href="https://blog.shunzi.tech/tag/javase/" class="ctag ctag-41 ctag-javase" aria-label="">JavaSE</a>
                    
                  </div>
                  <div class="clear"></div>
                </section>
              </div>

              <div class="grid-33 tablet-grid-50 mobile-grid-100">
                <section id="epcl_about-2" class="widget widget_epcl_about underline-effect">
                  <h4 class="widget-title title white bordered">关于我</h4>
                  <div class="avatar">
                    <a href="" class="translate-effect thumb"><span class="fullimage cover" style="background-image: url(https://blog.shunzi.tech/images/avatar.png);"></span></a>
                  </div>
                  <div class="info">
                    <h4 class="title small author-name gradient-effect no-margin"><a href="">Elvis Zhang</a></h4>
                    <p class="founder">The easy way or the right way.</p>
                    <div class="social">
                      
                          
                            <a href="https://github.com/zjs1224522500" class="translate-effect" target="_blank"><i class="fa fa-github"></i></a>
                        
                      
                          
                            <a href="https://twitter.com/1224522500Elvis" class="translate-effect" target="_blank"><i class="fa fa-twitter"></i></a>
                        
                      
                        
                      
                        
                      
                        
                      
                    </div> 
                  </div>
                  <div class="clear"></div>
                  </section>
              </div>

            </div>
            <div class="clear"></div>
        </div>

        <div class="logo">
          <a href="https://blog.shunzi.tech"><img src="\media\images\custom-footerLogo.jpg" alt=""></a>
        </div>
        <p class="published border-effect">
          ©2019 共 115 篇文章
          <br/>
          Theme <a href="https://gridea.dev/" target="_blank">「breek」</a> Powered by <a href="https://gridea.dev/" target="_blank">「Gridea」</a>
        </p>
        
        <a href="javascript:void(0)" id="back-to-top" class="epcl-button dark" style="display:none">
          <i class="fa fa-arrow"></i>
        </a>
    </footer>
    
    <div class="clear"></div>

        
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/leancloud-storage/dist/av-min.js"></script>
<script type="text/javascript" src="https://cdn.staticfile.org/valine/1.3.10/Valine.Pure.min.js"></script>
<script>
    new Valine({
        el: '#comments',
        appId: 'Pj5H1z0w7hJlLGJpGBh9NrCq-MdYXbMMI' ,
        appKey: 'LdR8vK5EaBfK87esF7tlbsXe',
        pageSize: 30,
        placeholder: '既然来了，那就留个痕迹吧~',
        visitor: true // 阅读量统计
    })
</script>
    

      
    <script src="https://blog.shunzi.tech/media/js/functions-post.js"></script>

    </div>
    <!-- end: #wrapper -->
  </body>
</html>
